/* This do-file cleans the raw data and produces the datasets used for analysis.
The cleaning is broken up into several steps because in some cases the
files do qualitatively different things (e.g. step 0,1,2) or because they can take 
considerable computing time/resources, e.g. using a Macbook Mini M4, 24 GB RAM
using STATA 15.1, it takes roughly 1 hour to run '3_firm_mn.do' because data is being
transformed, whereas '4_firm_mn_coding.do' takes at most a minute.
*/

clear
set memory 500m
set matsize 100
set more off 
# delimit;

* Set Directory; 

cd "~/Desktop/migrec_restat/slbfe_data/do/";


**************************************************************
*********  PART 1: PRODUCE AGENCY x MONTH LEVEL PANEL ********
**************************************************************;


* STEP 0: Read in raw csv files, do some basic cleaning and produce aggregated .dta files  ;
 
* Migrant data;
	do final_clean.do; 

* Job order data;
	do job_order_clean.do; 

* Complaints data;
	do complaints_clean.do; 

* Self migrate data;
	do self_clean.do; 

* Agency location data;
	do agency_location_clean.do; 

* Agency ratings data;
	do agency_ratings_clean.do; 

**************************************

* STEP 1: Aggregate .dta files from prior step to produce a firm x year x district
level dataset (unbalanced); 
	do 1_firm_mn_district.do; 

* STEP 2: Generate a balanced panel of the dataset produced in the prior step;	

	do 2_firm_mn_district_balanced.do;

**************************************
* STEP 3: merge in and collapse data;		

	do 3_firm_mn.do; 

**************************************
* STEP 4: create variables for analysis and clean data; 

	do 4_firm_mn_coding.do;
	
**************************************
* STEP 5: create final dataset for analysis, i.e. 5_firm_mn_final.dta;  		

	do 5_firm_mn_final.do;


**************************************************************
*********  PART 2: PRODUCE AGENCY x EMPLOYER LEVEL PANEL *****
**************************************************************;

* This dofile produces agency_employer_panel.dta ;


	do 6_employer_agency_panel.do; 
	
**************************************************************
*********  PART 3: CLEAN AGENCY SURVEY DATA ******************
**************************************************************;

* Remove PII from agency survey;

   do  7_agency_survey_clean.do;
	







